# -*- coding: utf-8 -*- 
#code by intosec(intosec@163.com)
import sys
import os
import re
import mmap
import datetime
import Queue
import threading
import getopt

reload(sys)
sys.setdefaultencoding('utf-8')

systype = sys.getdefaultencoding()

re_chopper = r'<html><head><meta http-equiv=content-type content="text/html; charset=UNICODE">' #first line in chopper html
re_getrows = r'<tr[\s\S]*?<tr>'  #Get a datarow from html
re_dict = { r'<tr.+?<td>' : '',
            r'</td><tr>' : '',
            r'</td><td>' : '\t',
            r'\r' : '<CR>', 
            r'\n' : '<LF>',
            r'\t' : '<TAB>'
           }  #Marks to be replaced in every datarow

queue = Queue.Queue()
MAX_THREAD = 1
mutex = threading.Lock()

def usage():
    print 'Usage:'
    print '    -h,--help    show this'
    print '    -f,--file    Convert a single HTML file from chopper'
    print '    -d,--dir     Convert multi HTML files from chopper in a directory'
    print '    Don\'t use -f and -d at the same time!'
    print '\nExample:'
    print '    chopperhtml2txt.py -f members.html'
    print '    chopperhtml2txt.py -d D:\\database\\'


def CheckFile(filepath):
    try:
        f = open(filepath)
        line = f.readline()
        f.close()
        m = cmp(line[:-3], re_chopper.encode('utf-16'))
        if m:
            return False
        else:
            return True
    except:
        return False

def GetFileList(directory):
    directory = directory.replace('"', '')
    if cmp(directory[-1], '\\'):
        d = directory + '\\'
    else:
        d = directory
    dir_list=os.listdir(d)
    file_list=[]
    print '[*]Searching HTML files...'
    for l in dir_list:
        if l.rfind('html')>=0:
            if CheckFile(d + l):
                file_list.append(d + l)
    print '[*]Found %s chopper html files' % len(file_list)
    return file_list

#Read HTML File And Decode
def LoadFile(filepath):
    if CheckFile(filepath):
        size = os.path.getsize(filepath)
        f = open(filepath, 'r+')
        m = mmap.mmap(f.fileno(), 0)
        f.close()
    else:
        sys.exit()
    
    data = m.read(size)
    data = data[0:size].decode('utf-16').encode(systype)
    return data


def GetSQL(fileName):
    f = open(fileName)
    line = ''
    for i in range(6):
        line = f.readline()
    line = repr(line).replace(r'\x00', '')
    return re.findall(r'(?<=<div>).+(?=</div>)', line)[0]


def GetRows(text):
    return re.findall(re_getrows, text)


def FormatRow(row):
    for it in re_dict:
        #print it + ':' + re_dict[it]
        row = re.sub(it, re_dict[it], row)
    return row


class ThreadFile(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.lock = threading.RLock()
        self.queue = queue

    def run(self):
        while True:
            #grabs host from queue
            f = self.queue.get()
            index = f.find('|', 0)
            fileNum = f[0:index]
            fileName = f[index+1:len(f)]
            ConvertFile(fileNum, fileName)
            #signals to queue job is done
            self.queue.task_done()


def ConvertFile(fileNum, fileName):
    num = str(fileNum)
    data = LoadFile(fileName)
    mutex.acquire()
    print '[*]File_%s:%s Loaded!' % (num, fileName.split('\\')[-1])
    mutex.release()
    sql = GetSQL(fileName)
    mutex.acquire()
    print '[*]File_%s:SQL Query:"%s"' % (num, sql)
    mutex.release()
    rows = GetRows(data)
    mutex.acquire()
    print '[*]File_%s:Found %s rows!' % (num, len(rows))
    mutex.release()

    wf = open(fileName.replace('.html', '.txt'), 'w')
    wf.write('#%s\n\n' % sql)
    for row in rows:
        wf.write(FormatRow(row))
        wf.write('\n')
    wf.close()
    mutex.acquire()
    print '[*]File_%s:TXT saved in %s' % (num, fileName.replace('.html', '.txt'))
    mutex.release()

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hf:d:', ['help', 'file', 'dir'])
    except getopt.GetoptError:
        usage()
        print '1'
        sys.exit()
    
    if len(opts)== 0:
        usage()
        sys.exit()
    
    file_list = []
    threads_num = 0
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-f','--file'):
            if CheckFile(arg):
                file_list.append(arg)
            threads_num = 1
            break
        elif opt in ('-d','--dir'):
            file_list = GetFileList(arg)
            threads_num = MAX_THREAD
            break
        else:
            usage()
            sys.exit()
    
    if len(file_list) == 0:
        print '[!]Check your files!'
        sys.exit()
    
    start = datetime.datetime.now()

    for i in range(threads_num):
        t = ThreadFile(queue)
        t.setDaemon(True)
        t.start()
        
    i = 1
    for f in file_list:
        queue.put(str(i) + '|' + f)
        i = i + 1
    #wait on the queue until everything has been processed     
    queue.join()
    
    end = datetime.datetime.now()
    print "[*]Elapsed Time: %s" % (end - start)
    print '[*]Done!\n'
        
main()
